NOW EXPERIMENTING:¶
Pinet:
- check iou_metrix function
- try SGD, its noiser
- set higher w = 1.0, ramp up delay to 50, ramp up peroid to 50, only sample 3600 test dataset
- change core model
Done:¶
- regularized by pseudo labeled data, but it also have some lagecy label which seems in the eraly stage, seems wrong
- try pure training dataset, should be same to normal train
- noise and rotation cause huge performance decrease.....
- randomize in first batch otherwise loss seems a bit strange.
- add some random noice
- custom data generate to update test temperal mask every n epoch
- change unsupervise loss weight on the fly
- used full model (without dropout) for validation and prediction
- close augmentation: noise, rotation, no temperal predict, no temperal loss
- ramp-up function is also important as early psuedo label is very wrong, reduced temperal loss in early stage, otherwise network will learn wrong pattern from early stage psuedo label
- train on train then test order, insure frist part of training has all label, which won't confuse network, solution: use a ration to adjust how much label vs unlabel loss will go into total loss (unlabled size / batch size) * w
- loss is toooooooo high without any reason: its cause by test data, when update weight with test data which don't have label, it cause network confuse because those update don't have gradient, (are based on other's graident)...
- sample data(train + test) from db, split train data into train val data
- apply test data mask if its none (no temperal mask)
- basic augmentation twice for pinet
- custom pi net loss function
- move prediction into here as it used custom function
work:¶
- image data augmentation, flip, crop,
- resnet with high dropout, (as resnet so easy overfitting, and not enough data)
- 2d spatial dropout
- dice loss, faster converge, but doesn't help improve score....
not work:¶
- deeper, shellower
- Clahe
- pure dice loss, it will give binary solution, rather than probability
- dropout=0.6, overfitting, Val=0.8, LB=0.77
idea:¶
- augmentation: rotation, affine transform, Elastic deformations
- reduce dropout, as always under fitting
- smaller batch size
- CRF
- inception block
In [ ]:
iou_thresholds = np.array([0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95])
def iou(img_true, img_pred):
i = np.sum((img_true*img_pred) >0)
u = np.sum((img_true + img_pred) >0)
if u == 0:
return u
return i/u
def iou_metric(imgs_true, imgs_pred):
num_images = len(imgs_true)
scores = np.zeros(num_images)
for i in range(num_images):
if imgs_true[i].sum() == imgs_pred[i].sum() == 0:
scores[i] = 1
else:
scores[i] = (iou_thresholds <= iou(imgs_true[i], imgs_pred[i])).mean()
return scores.mean()
In [1]:
import numpy as np
import pandas as pd
import time, sys, json, os, io, cv2, base64
from io import BytesIO
from subprocess import check_output
from pprint import pprint as pp
import pymongo
from pymongo import MongoClient
import hashlib
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
import matplotlib.pyplot as plt
from imageio import imread
from skimage.transform import resize
from sklearn.model_selection import train_test_split
from random import randint
from skimage import exposure
from skimage.color import gray2rgb
from skimage.color import rgb2gray
from skimage import transform
from skimage import util
from tqdm import tqdm_notebook
import skimage as sk
import gc
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import unary_from_labels, create_pairwise_bilateral
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
os.environ["CUDA_VISIBLE_DEVICES"]="0"
import matplotlib.pyplot as plt
plt.style.use('seaborn-white')
import seaborn as sns
sns.set_style("white")
import keras
from keras.preprocessing.image import load_img
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.models import load_model
from keras.optimizers import Adam
from keras.utils.vis_utils import plot_model
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, concatenate, Dropout, BatchNormalization
from keras.layers.core import Lambda, RepeatVector, Reshape
from keras.models import Model, load_model
from keras.layers import Input, UpSampling2D, Concatenate, SpatialDropout2D
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras import backend as K
from keras.backend.tensorflow_backend import set_session
from keras.layers.merge import add
from keras import regularizers
from keras.regularizers import l2
from keras.losses import binary_crossentropy
from keras.activations import softmax
from keras.backend import tensorflow_backend, common
from keras.callbacks import LambdaCallback, Callback
import tensorflow as tf
%load_ext autoreload
%autoreload 2
In [2]:
path_train = '../data/train/'
path_test = '../data/test/'
train_ids = next(os.walk(path_train))[2]
test_ids = next(os.walk(path_test))[2]
depths_df = pd.read_csv("../data/depths.csv", index_col="id")
train_df = pd.read_csv("../data/train.csv", index_col="id")
In [3]:
def _connect_mongo(host, port, username, password, db):
""" A util for making a connection to mongo """
if username and password:
mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host, port, db)
conn = MongoClient(mongo_uri)
else:
conn = MongoClient(host, port)
return conn[db]
def insert_data(data, db, collection, check_id='id', host='localhost', port=27017, username=None, password=None, no_id=True):
db = _connect_mongo(host=host, port=port, username=username, password=password, db=db)
duplicate_result = db[collection].find(
{check_id: data[check_id]})
print('duplicate count' + str(duplicate_result.count()))
if duplicate_result.count() == 0:
db[collection].insert_one(data)
def set_data(db, collection, search, set_query, host='localhost', port=27017, username=None, password=None, no_id=True):
db = _connect_mongo(host=host, port=port, username=username, password=password, db=db)
db[collection].update(
search,
{ '$set': set_query }
)
def read_mongo(db, collection, query={}, host='localhost', port=27017, username=None, password=None, no_id=True):
""" Read from Mongo and Store into DataFrame """
# Connect to MongoDB
db = _connect_mongo(host=host, port=port, username=username, password=password, db=db)
# Make a query to the specific DB and Collection
cursor = db[collection].find(query)
# Expand the cursor and construct the DataFrame
df = pd.DataFrame(list(cursor))
# Delete the _id
if no_id:
del df['_id']
return df
def sample_mongo(db, collection, query={}, host='localhost', port=27017, username=None, password=None, no_id=True, num_sample=1000):
""" Read from Mongo and Store into DataFrame """
# Connect to MongoDB
db = _connect_mongo(host=host, port=port, username=username, password=password, db=db)
# Make a query to the specific DB and Collection
cursor = db[collection].aggregate([{ "$sample": { "size": num_sample }}], allowDiskUse=True)
# Expand the cursor and construct the DataFrame
df = pd.DataFrame(list(cursor))
# Delete the _id
if no_id:
del df['_id']
return df
Put all dataset into DF frist, hold in memory¶
In [4]:
img_size_ori = 101
img_size_target = 128
def read_resize_img(x, scale,mask=False):
img_byte = io.BytesIO(base64.b64decode(x))
img_np = np.array(imread(img_byte, as_gray=True))/scale
img_np = resize(img_np,(img_size_target,img_size_target), mode='constant', preserve_range=True)
if mask:
img_np[img_np>0] = 1
return img_np
train_df = read_mongo('dataset', 'tgs_salt', {"$and": [{"img_mask_base64":{"$ne":None}}]})
############# sample or all test data from db #############
# test_df = read_mongo('dataset', 'tgs_salt', {"$and": [{"img_mask_base64":{"$eq":None}}]})
test_df = sample_mongo('dataset', 'tgs_salt', {"$and": [{"img_mask_base64":{"$eq":None}}]}, num_sample=3600)
train_df['img'] = train_df['img_base64'].apply(lambda x: read_resize_img(x, 256.0))
train_df['img_mask'] = train_df['img_mask_base64'].apply(lambda x: read_resize_img(x, 65535.0, mask=True))
train_df['img_temperal_mask'] = train_df['img'].apply(lambda x: -1*np.ones((img_size_target,img_size_target)))
test_df['img'] = test_df['img_base64'].apply(lambda x: read_resize_img(x, 256.0))
test_df['img_mask'] = test_df['img'].apply(lambda x: -1*np.ones((img_size_target,img_size_target)))
test_df['img_temperal_mask'] = test_df['img'].apply(lambda x: -1*np.ones((img_size_target,img_size_target)))
train_df = train_df.drop('img_base64', axis=1)
train_df = train_df.drop('img_mask_base64', axis=1)
train_df = train_df.drop('ASM', axis=1)
train_df = train_df.drop('contrast', axis=1)
train_df = train_df.drop('correlation', axis=1)
train_df = train_df.drop('dissimilarity', axis=1)
train_df = train_df.drop('energy', axis=1)
train_df = train_df.drop('entropy', axis=1)
train_df = train_df.drop('homogeneity', axis=1)
train_df = train_df.drop('coverage', axis=1)
train_df = train_df.drop('coverage_class', axis=1)
test_df = test_df.drop('img_base64', axis=1)
# test_df = test_df.drop('img_mask_base64', axis=1)
# test_df = test_df.drop('ASM', axis=1)
# test_df = test_df.drop('contrast', axis=1)
# test_df = test_df.drop('correlation', axis=1)
# test_df = test_df.drop('dissimilarity', axis=1)
# test_df = test_df.drop('energy', axis=1)
# test_df = test_df.drop('entropy', axis=1)
# test_df = test_df.drop('homogeneity', axis=1)
# test_df = test_df.drop('coverage', axis=1)
# test_df = test_df.drop('coverage_class', axis=1)
train_df, val_df = train_test_split(train_df, test_size=0.1)
############# use test + train or only train data #############
train_test_df = pd.concat([train_df, test_df], axis=0, ignore_index=True, sort=False).sample(frac=1).reset_index(drop=True)
# train_test_df = train_df.copy()
print(train_df.shape)
print(val_df.shape)
print(test_df.shape)
print(train_test_df.shape)
In [5]:
# train_test_df.head(-1)
Data Preprcessing function: sample, augment, to np_array¶
In [6]:
def noisy(noise_typ,image):
if noise_typ == "gauss":
row,col,ch= image.shape
mean = 0
var = 0.0005
sigma = var**0.5
gauss = np.random.normal(mean,sigma,(row,col,ch))
gauss = gauss.reshape(row,col,ch)
noisy = image + gauss
return noisy
elif noise_typ == "s&p":
row,col,ch = image.shape
s_vs_p = 0.5
amount = 0.01
out = np.copy(image)
# Salt mode
num_salt = np.ceil(amount * image.size * s_vs_p)
coords = [np.random.randint(0, i, int(num_salt))
for i in image.shape]
out[coords] = 1
# Pepper mode
num_pepper = np.ceil(amount* image.size * (1. - s_vs_p))
coords = [np.random.randint(0, i, int(num_pepper))
for i in image.shape]
out[coords] = 0
return out
elif noise_typ == "poisson":
vals = len(np.unique(image))
vals = 2 ** np.ceil(np.log2(vals))
noisy = np.random.poisson(image * vals) / float(vals)
return noisy
elif noise_typ =="speckle":
scale = 0.05
row,col,ch = image.shape
gauss = np.random.randn(row,col,ch)
gauss = gauss.reshape(row,col,ch)
noisy = image + image * gauss * scale
return noisy
elif noise_typ == 'None':
return image
def _img_augmentation(_df, if_augment=True):
def random_crop_resize(x, crop, flip, degree, noise_type='None'):
# return x if all pixel is -1
if np.mean(x) == -1:
return x
x = np.fliplr(x) if flip else x
######### noise, rotate decrease the performance a lot... ###########
# x = np.squeeze(noisy(noise_type, np.expand_dims(x, axis=3)))
# x = sk.transform.rotate(x, degree)
x = x[crop[0]:-crop[1],crop[2]:-crop[3]]
x = resize(x,(img_size_target,img_size_target), mode='constant', preserve_range=True)
return x
def img_augment(df):
augment_df = pd.DataFrame()
for index, row in df.iterrows():
# np.random.seed(0)
crop = np.random.randint(low=1, high=30, size=4) if if_augment else np.array([0, 1, 0, 1])
flip = np.random.choice([True, False]) if if_augment else False
degree = np.random.uniform(-10, 10) if if_augment else 0
noise_type = np.random.choice(['gauss', 'poisson', 's&p', 'speckle', 'None']) if if_augment else 'None'
aug_img = random_crop_resize(row['img'], crop, flip, degree, noise_type)
aug_img_mask = random_crop_resize(row['img_mask'], crop, flip, degree)
aug_img_temperal_mask = random_crop_resize(row['img_temperal_mask'], crop, flip, degree)
augment_df = augment_df.append(
{
'depth': row['depth'],
'img_id': row['img_id']+'_augment',
'aug_img': aug_img,
'aug_img_mask': aug_img_mask,
'aug_img_temperal_mask': aug_img_temperal_mask,
}, ignore_index=True
)
return augment_df
_augment_df = img_augment(_df)
return _augment_df
def _convert_to_np_array(_augment_df):
X_np = np.expand_dims(np.stack((np.asarray(_augment_df['aug_img'].values.tolist()))),axis=3)
y_np = np.expand_dims(np.asarray(_augment_df['aug_img_mask'].values.tolist()),axis=3)
y_temp_np = np.expand_dims(np.asarray(_augment_df['aug_img_temperal_mask'].values.tolist()),axis=3)
y_np = np.concatenate((y_np,y_temp_np),axis=3)
return X_np, y_np
def calculate_temperal_mask(epoch):
global model_train, train_df, test_df, graph_train
with graph_train.as_default():
X_train = np.expand_dims(np.stack((np.asarray(train_df['img'].values.tolist()))),axis=3)
predict_train = model_train.predict(X_train,batch_size=64, verbose=1)
predict_train = np.squeeze(predict_train)
X_test = np.expand_dims(np.stack((np.asarray(test_df['img'].values.tolist()))),axis=3)
predict_test = model_train.predict(X_test,batch_size=64, verbose=1)
predict_test = np.squeeze(predict_test)
idx = 0;
alpha = 0.5
# temperal_mask_ramp_up = 1/(1-(alpha**(epoch+1)))
for index, row in tqdm_notebook(train_df.iterrows(),total=len(train_df.index)):
img_temperal_mask = row['img_temperal_mask']
predict = predict_train[idx]; idx+=1;
if(np.mean(img_temperal_mask) < 0):
train_df.at[index,'img_temperal_mask'] = predict
else:
train_df.at[index,'img_temperal_mask'] = (img_temperal_mask*alpha + predict*(1-alpha))
for index, row in tqdm_notebook(test_df.iterrows(),total=len(test_df.index)):
img_temperal_mask = row['img_temperal_mask']
predict = predict_test[index]
if(np.mean(img_temperal_mask) < 0):
test_df.at[index,'img_temperal_mask'] = predict
else:
test_df.at[index,'img_temperal_mask'] = (img_temperal_mask*alpha + predict*(1-alpha))
train_test_df = pd.concat([train_df, test_df], axis=0, ignore_index=True, sort=False).sample(frac=1).reset_index(drop=True)
return train_test_df
plot some augmented image¶
In [7]:
# sample_train_df = train_df.sample(50)
# train_augment_df = _img_augmentation(sample_train_df)
# %%time
# sample_train_df, sample_val_df, sample_test_df = sample_df(train_df, val_df, test_df)
# train_augment_df, val_augment_df, test_augment_df = img_augmentation(sample_train_df, sample_val_df, sample_test_df)
# X_train, y_train, X_valid, y_valid = convert_to_np_array(train_augment_df, val_augment_df, test_augment_df)
In [8]:
# test_augment_df.head(1)
# train_augment_df.head(10)
In [9]:
# base_idx = 0
# max_images = 16
# grid_width = 4
# grid_height = int(max_images / grid_width)
# fig, axs = plt.subplots(grid_height, grid_width, figsize=(20, 20))
# row = 0; col = 0;
# for i, idx in enumerate(train_augment_df.index[base_idx:base_idx+int(max_images)]):
# img = train_augment_df.loc[idx].aug_img
# mask = train_augment_df.loc[idx].aug_img_mask
# ax = axs[row, col];
# ax.imshow(img, cmap="seismic")
# # ax.imshow(img, cmap="gray")
# ax.imshow(mask, alpha=0.8, cmap="Reds"); col+=1;
# ax.set_yticklabels([]); ax.set_xticklabels([]);
# if col >= grid_width:
# col=0; row+=1;
Custom loss function¶
In [10]:
from debug import _debug_func
# Define IoU metric
def mean_iou(y_true, y_pred):
prec = []
for t in np.arange(0.5, 1.0, 0.05):
y_pred_ = tf.to_int32(y_pred > t)
score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)
K.get_session().run(tf.local_variables_initializer())
with tf.control_dependencies([up_opt]):
score = tf.identity(score)
prec.append(score)
return K.mean(K.stack(prec), axis=0)
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred = K.cast(y_pred, 'float32')
y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
intersection = y_true_f * y_pred_f
score = 2. * K.sum(intersection) / (K.sum(y_true_f) + K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = y_true_f * y_pred_f
score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
return 1. - score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)
def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred) - K.log(1. - dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1. - epsilon)
logit_y_pred = K.log(y_pred / (1. - y_pred))
loss = weight * (logit_y_pred * (1. - y_true) +
K.log(1. + K.exp(-K.abs(logit_y_pred))) + K.maximum(-logit_y_pred, 0.))
return K.sum(loss) / K.sum(weight)
def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
w, m1, m2 = weight, y_true, y_pred
intersection = (m1 * m2)
score = (2. * K.sum(w * intersection) + smooth) / (K.sum(w * m1) + K.sum(w * m2) + smooth)
loss = 1. - K.sum(score)
return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
y_pred = K.cast(y_pred, 'float32')
# if we want to get same size of output, kernel size must be odd
averaged_mask = K.pool2d(
y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
weight = K.ones_like(averaged_mask)
w0 = K.sum(weight)
weight = 5. * K.exp(-5. * K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
weight *= (w0 / w1)
loss = weighted_bce_loss(y_true, y_pred, weight) + dice_loss(y_true, y_pred)
return loss
# MSE between current and temporal outputs
def temperal_mse_loss(y_true, y_pred):
def mse_loss(y_pred, y_temperal, temperal_size):
# generate filter using y_temperal
temperal_filter = tf.stop_gradient(tf.cast(tf.not_equal(y_temperal, -1), tf.float32))
# filter out MSE if temperal = -1
quad_diff = K.sum((temperal_filter*y_pred - temperal_filter*y_temperal) ** 2) / (temperal_size*128*128+1e-15)
return quad_diff
y_temperal = tf.slice(y_true, [0, 0, 0, 1], [-1, -1, -1, 1])
# count temperal size which has value (not -1)
temperal_size=tf.stop_gradient(tf.reduce_sum(tf.cast(tf.not_equal(tf.reduce_min(y_temperal, [1,2,3]), -1), tf.float32)))
quad_diff = tf.cond(temperal_size > 0, lambda: mse_loss(y_pred, y_temperal, temperal_size), lambda: 0.0)
return quad_diff
def masked_crossentropy(y_true, y_pred):
y_mask = tf.slice(y_true, [0, 0, 0, 0], [-1, -1, -1, 1])
# y_mask = _debug_func((y_mask),"y_mask")
# count mask size
mask_size=tf.stop_gradient(tf.reduce_sum(tf.cast(tf.not_equal(tf.reduce_min(y_mask, [1,2,3]), -1), tf.float32)))
# generate mask filter as test doesn't have mask (-1)
mask_filter = tf.stop_gradient(tf.cast(tf.not_equal(y_mask, -1), tf.float32))
y_mask = y_mask*mask_filter
y_pred = y_pred*mask_filter
bce_loss_ = tf.cond(mask_size > 0, lambda: (K.sum(binary_crossentropy(y_mask, y_pred))/(mask_size*128*128+1e-15)), lambda: 0.0)
dice_loss_ = dice_loss(y_mask, y_pred)
bce_dice_loss_ = bce_loss_ + dice_loss_
return bce_dice_loss_
def mask_mean_iou(y_true, y_pred):
y_mask = tf.slice(y_true, [0, 0, 0, 0], [-1, -1, -1, 1])
mask_filter = tf.cast(tf.not_equal(y_mask, -1), tf.float32)
mask_mean_iou = mean_iou(mask_filter*y_mask, mask_filter*y_pred)
return mask_mean_iou
def temperal_mean_iou(y_true, y_pred):
y_temperal = tf.slice(y_true, [0, 0, 0, 1], [-1, -1, -1, 1])
temperal_filter = tf.cast(tf.not_equal(y_temperal, -1), tf.float32)
temperal_mean_iou = mean_iou(temperal_filter*(tf.cast(tf.greater(y_temperal,0.5), tf.float32)), temperal_filter*y_pred)
return temperal_mean_iou
class Temperal_Callback(Callback):
def __init__(self, delay_temperal=0, ramp_up_period=50):
self.ramp_up = 0
self.ramp_up_period = ramp_up_period
self.w = 1.0
self.delay_temperal = delay_temperal
# customize your behavior
def on_epoch_end(self, epoch, logs={}):
if epoch-self.delay_temperal < 0:
self.ramp_up = 0
elif epoch-self.delay_temperal < self.ramp_up_period:
self.ramp_up = np.exp(-5*((1-((epoch-self.delay_temperal)/self.ramp_up_period)))**2)
else:
self.ramp_up = 1
logger.info("epoch %s, ramp_up = %s" % (epoch, self.ramp_up))
def temporal_loss(self, y_true, y_pred):
sup_loss = masked_crossentropy(y_true, y_pred)
unsup_loss = temperal_mse_loss(y_true, y_pred)
batch_size = tf.stop_gradient(tf.to_float(tf.shape(y_pred)[0]))
y_mask = tf.stop_gradient(tf.slice(y_true, [0, 0, 0, 0], [-1, -1, -1, 1]))
mask_size = tf.stop_gradient(tf.reduce_sum(tf.cast(tf.not_equal(tf.reduce_min(y_mask, [1,2,3]), -1), tf.float32)))
w = self.w * (1.0-(mask_size/batch_size)) * self.ramp_up
# sup_loss = _debug_func((sup_loss),"sup_loss")
# unsup_loss = _debug_func((unsup_loss),"unsup_loss")
######## use sup+unsup or only sup loss########
# return sup_loss
return sup_loss + w * unsup_loss
DataGenerator¶
In [11]:
class DataGenerator(keras.utils.Sequence):
#class static variable
global model_train
_val_df = None
_train_test_df = None
_X_train = None
_y_train = None
_X_valid = None
_y_valid = None
'Generates data for Keras'
def __init__(self, train_test_df, val_df,
batch_size=32, shuffle=True, training=True, temperal_every_epoch=1, delay_temperal=0):
self.training = training
self.batch_size = batch_size
self.shuffle = shuffle
self.epoch = 0
self.val_size = val_df.shape[0]
self.train_test_size = train_test_df.shape[0]
self.temperal_every_epoch = temperal_every_epoch
self.delay_temperal = delay_temperal
print(f'val_size = {self.val_size}, \
train_test_size = {self.train_test_size}, \
')
'Initial static variable'
if self.training:
DataGenerator._val_df = val_df
DataGenerator._train_test_df = train_test_df
def __len__(self):
'Denotes the number of batches per epoch'
if self.training:
print('train lenght')
print(int(np.floor((self.train_test_size) / self.batch_size)))
return int(np.floor((self.train_test_size) / self.batch_size))
else:
print('val lenght')
print(int(np.floor(self.val_size / self.batch_size)))
return int(np.floor(self.val_size / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
'index: indicate which batch in a epoch, (first, second or third batch)'
# Generate indexes of the batch
if self.training:
batch_train_df = DataGenerator._train_test_df.iloc[index*self.batch_size:(index+1)*self.batch_size]
train_augment_df = _img_augmentation(batch_train_df)
X_train_np, y_train_np = _convert_to_np_array(train_augment_df)
return X_train_np, y_train_np
else:
batch_val_df = DataGenerator._val_df.iloc[index*self.batch_size:(index+1)*self.batch_size]
val_augment_df = _img_augmentation(batch_val_df, if_augment=False)
X_val_np, y_val_np = _convert_to_np_array(val_augment_df)
del batch_val_df, val_augment_df
return X_val_np, y_val_np
def on_epoch_end(self):
'Updates indexes after each epoch'
print(f', epoch={self.epoch}')
if self.epoch % self.temperal_every_epoch == 0 and self.training and self.epoch >= self.delay_temperal:
new_train_test_df = calculate_temperal_mask(self.epoch)
DataGenerator._train_test_df = new_train_test_df
self.epoch += 1
Build Unet + Resnet¶
In [12]:
# tensorflow session setting
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
set_session(tf.Session(config=tf_config))
def conv_block(m, dim, acti, bn, res, do=0, training=None):
n = Conv2D(dim, 3, activation=acti, padding='same')(m)
n = BatchNormalization()(n) if bn else n
n = SpatialDropout2D(do/2.0)(n, training=training) if do else n
n = Conv2D(dim, 3, activation=acti, padding='same')(n)
n = BatchNormalization()(n) if bn else n
n = Concatenate()([m, n]) if res else n
n = SpatialDropout2D(do)(n, training=training) if do else n
return n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res,training=None):
if depth > 0:
n = conv_block(m, dim, acti, bn, res, training=training)
m = MaxPooling2D()(n) if mp else Conv2D(dim, 3, strides=2, padding='same')(n)
m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res, training=training)
if up:
m = UpSampling2D()(m)
m = Conv2D(dim, 2, activation=acti, padding='same')(m)
else:
m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same')(m)
n = Concatenate()([n, m])
m = conv_block(n, dim, acti, bn, res, training=training)
else:
m = conv_block(m, dim, acti, bn, res, do, training=training)
return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=5, inc_rate=2., activation='relu',
dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=True, training=None):
i = Input(shape=img_shape)
o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual, training=training)
o = Conv2D(out_ch, 1, activation='sigmoid')(o)
return Model(inputs=i, outputs=o)
In [13]:
# used for training unsuperivsed, that keep dropout
global model_train, graph_train
epochs = 100
batch_size = 32
dropout = 0.6
####### control when to start temperal ensemble #############
delay_temperal = epochs/2
####### control how many epoch that ramp up from 0 to 1 #########
ramp_up_period = epochs/2
temperal = Temperal_Callback(delay_temperal=delay_temperal, ramp_up_period=ramp_up_period)
model_train = UNet((img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True, dropout=dropout, training=True)
model_train.compile(loss=temperal.temporal_loss, optimizer=keras.optimizers.Adam(), metrics=[masked_crossentropy, temperal_mse_loss, mask_mean_iou, temperal_mean_iou])
model_train.summary()
graph_train = tf.get_default_graph()
In [14]:
callbacks = [
temperal,
EarlyStopping(patience=10, verbose=1, monitor="loss", mode="min"),
ReduceLROnPlateau(factor=0.5, patience=5, min_lr=0.00001, verbose=1),
ModelCheckpoint('model-u-res-pi-net.h5', verbose=1, save_best_only=True, monitor="loss", mode="min"),
ModelCheckpoint('weight-u-res-pi-net.h5', verbose=1, save_best_only=True, monitor="loss", mode="min", save_weights_only=True),
]
training_generator = DataGenerator( train_test_df, val_df, batch_size=batch_size, training=True, delay_temperal=delay_temperal)
validation_generator = DataGenerator( train_test_df, val_df, batch_size=batch_size, training=False, delay_temperal=delay_temperal)
history = model_train.fit_generator(generator=training_generator,
validation_data=validation_generator,
epochs=epochs, callbacks=callbacks,
use_multiprocessing=True,
workers=4)
In [15]:
fig, (ax_loss, ax_temp_loss, ax_acc, ax_iou) = plt.subplots(1,4, figsize=(15,5))
ax_loss.plot(history.epoch, history.history["loss"], label="Train loss")
ax_loss.plot(history.epoch, history.history["val_loss"], label="Validation loss")
ax_temp_loss.plot(history.epoch, history.history["temperal_mse_loss"], label="Train loss")
# ax_temp_loss.plot(history.epoch, history.history["val_temperal_mse_loss"], label="Validation loss")
ax_acc.plot(history.epoch, history.history["mask_mean_iou"], label="Train mask iou")
ax_acc.plot(history.epoch, history.history["val_mask_mean_iou"], label="Validation mask iou")
ax_iou.plot(history.epoch, history.history["temperal_mean_iou"], label="Train temperal iou")
ax_iou.plot(history.epoch, history.history["val_temperal_mean_iou"], label="Validation temperal iou")
Out[15]:
Fine tune threshold¶
In [16]:
# model = load_model("./model-unet-resnet.h5", custom_objects={'mean_iou':mean_iou})
# # used for predict, no dropout
temperal = Temperal_Callback()
model_predict = UNet((img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True, dropout=0.0, training=False)
model_predict.compile(loss=temperal.temporal_loss, optimizer="adam", metrics=[masked_crossentropy, temperal_mse_loss, mask_mean_iou, temperal_mean_iou])
model_predict.set_weights(model_train.get_weights())
X_valid = np.expand_dims(np.stack((np.asarray(val_df['img'].values.tolist()))),axis=3)
y_valid = np.expand_dims(np.asarray(val_df['img_mask'].values.tolist()),axis=3)
preds_valid = model_predict.predict(X_valid, batch_size=32, verbose=1)
In [17]:
# plot some validate result
base_idx = 10
max_images = 32
grid_width = 4
grid_height = int(max_images / grid_width)
fig, axs = plt.subplots(grid_height, grid_width, figsize=(20, 20))
row = 0; col = 0;
for i, idx in enumerate(val_df.index[base_idx:base_idx+int(max_images/2)]):
img = val_df.iloc[i].img
mask = val_df.iloc[i].img_mask
pred = preds_valid[i].squeeze()
ax = axs[row, col];
ax.imshow(img, cmap="seismic")
ax.imshow(mask, alpha=0.8, cmap="Reds"); col+=1;
ax.set_yticklabels([]); ax.set_xticklabels([]);
ax = axs[row, col];
ax.imshow(img, cmap="seismic")
ax.imshow(pred, alpha=0.8, cmap="Reds"); col+=1;
ax.set_yticklabels([]);ax.set_xticklabels([]);
if col >= grid_width:
col=0; row+=1;
In [18]:
# plot some temperal mask on test results
base_idx = 0
max_images = 16
grid_width = 4
grid_height = int(max_images / grid_width)
fig, axs = plt.subplots(grid_height, grid_width, figsize=(20, 20))
row = 0; col = 0;
for i, idx in enumerate(test_df.index[base_idx:base_idx+int(max_images)]):
img = test_df.loc[idx].img
mask = test_df.loc[idx].img_temperal_mask
ax = axs[row, col];
ax.imshow(img, cmap="seismic")
# ax.imshow(img, cmap="gray")
ax.imshow(mask, alpha=0.8, cmap="Reds"); col+=1;
ax.set_yticklabels([]); ax.set_xticklabels([]);
if col >= grid_width:
col=0; row+=1;
In [19]:
# src: https://www.kaggle.com/aglotero/another-iou-metric
def iou_metric(y_true_in, y_pred_in, print_table=False):
labels = y_true_in
y_pred = y_pred_in
true_objects = 2
pred_objects = 2
intersection = np.histogram2d(labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects))[0]
# Compute areas (needed for finding the union between all objects)
area_true = np.histogram(labels, bins = true_objects)[0]
area_pred = np.histogram(y_pred, bins = pred_objects)[0]
area_true = np.expand_dims(area_true, -1)
area_pred = np.expand_dims(area_pred, 0)
# Compute union
union = area_true + area_pred - intersection
# Exclude background from the analysis
intersection = intersection[1:,1:]
union = union[1:,1:]
union[union == 0] = 1e-9
# Compute the intersection over union
iou = intersection / union
# Precision helper function
def precision_at(threshold, iou):
matches = iou > threshold
true_positives = np.sum(matches, axis=1) == 1 # Correct objects
false_positives = np.sum(matches, axis=0) == 0 # Missed objects
false_negatives = np.sum(matches, axis=1) == 0 # Extra objects
tp, fp, fn = np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)
return tp, fp, fn
# Loop over IoU thresholds
prec = []
if print_table:
print("Thresh\tTP\tFP\tFN\tPrec.")
for t in np.arange(0.5, 1.0, 0.05):
tp, fp, fn = precision_at(t, iou)
if (tp + fp + fn) > 0:
p = tp / (tp + fp + fn)
else:
p = 0
if print_table:
print("{:1.3f}\t{}\t{}\t{}\t{:1.3f}".format(t, tp, fp, fn, p))
prec.append(p)
if print_table:
print("AP\t-\t-\t-\t{:1.3f}".format(np.mean(prec)))
return np.mean(prec)
def iou_metric_batch(y_true_in, y_pred_in):
batch_size = y_true_in.shape[0]
metric = []
for batch in range(batch_size):
value = iou_metric(y_true_in[batch], y_pred_in[batch],print_table=False)
metric.append(value)
return np.mean(metric)
from math import sqrt
from joblib import Parallel, delayed
import multiprocessing
from tqdm import tqdm
thresholds = np.linspace(0, 1, 20)
# result = Parallel(n_jobs=2)(io_metric_batch(train_y, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds))
# ious = np.array([iou_metric_batch(train_y, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)])
ious = np.array([iou_metric_batch(y_valid, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)])
In [20]:
threshold_best_index = np.argmax(ious)
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
print(f'threshold_best: {threshold_best}, iou_best: {iou_best}')
plt.plot(thresholds, ious)
plt.plot(threshold_best, iou_best, "xr", label="Best threshold")
plt.xlabel("Threshold")
plt.ylabel("IoU")
plt.title("Threshold vs IoU ({}, {})".format(threshold_best, iou_best))
plt.legend()
Out[20]:
Predict test data¶
In [21]:
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img, (img_size_target, img_size_target), mode='constant', preserve_range=True)
#res = np.zeros((img_size_target, img_size_target), dtype=img.dtype)
#res[:img_size_ori, :img_size_ori] = img
#return res
def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img, (img_size_ori, img_size_ori), mode='constant', preserve_range=True)
#return img[:img_size_ori, :img_size_ori]
# Source https://www.kaggle.com/bguberfain/unet-with-depth
def RLenc(img, order='F', format=True):
"""
img is binary mask image, shape (r,c)
order is down-then-right, i.e. Fortran
format determines if the order needs to be preformatted (according to submission rules) or not
returns run length as an array or string (if format is True)
"""
bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
runs = [] ## list of run lengths
r = 0 ## the current run length
pos = 1 ## count starts from 1 per WK
for c in bytes:
if (c == 0):
if r != 0:
runs.append((pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
# if last run is unsaved (i.e. data ends with 1)
if r != 0:
runs.append((pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
return z[:-1]
else:
return runs
In [22]:
X_test = np.expand_dims(np.stack((np.asarray(test_df['img'].values.tolist()))),axis=3)
preds_test = model_predict.predict(X_test, batch_size=32, verbose=1)
final_preds_test = preds_test > threshold_best
In [23]:
base_idx = 160
max_images = 32
grid_width = 4
grid_height = int(max_images / grid_width)
fig, axs = plt.subplots(grid_height, grid_width, figsize=(20, 20))
row = 0; col = 0;
for i in range(base_idx,base_idx+int(max_images)):
img = X_test[i].squeeze()
mask = preds_test[i].squeeze()
ax = axs[row, col];
ax.imshow(img, cmap="seismic")
ax.imshow(mask, alpha=0.5, cmap="Reds"); col+=1;
ax.set_yticklabels([]); ax.set_xticklabels([]);
if col >= grid_width:
col=0; row+=1;
In [24]:
base_idx = 160
max_images = 32
grid_width = 4
grid_height = int(max_images / grid_width)
fig, axs = plt.subplots(grid_height, grid_width, figsize=(20, 20))
row = 0; col = 0;
for i in range(base_idx,base_idx+int(max_images)):
img = X_test[i].squeeze()
mask = final_preds_test[i].squeeze()
ax = axs[row, col];
ax.imshow(img, cmap="seismic")
ax.imshow(mask, alpha=0.5, cmap="Reds"); col+=1;
ax.set_yticklabels([]); ax.set_xticklabels([]);
if col >= grid_width:
col=0; row+=1;
Apply CRF¶
In [25]:
#Original_image = Image which has to labelled
#Mask image = Which has been labelled by some technique..
def crf(original_image, mask_img):
# Converting annotated image to RGB if it is Gray scale
if(len(mask_img.shape)<3):
mask_img = gray2rgb(mask_img)
# #Converting the annotations RGB color to single 32 bit integer
annotated_label = mask_img[:,:,0] + (mask_img[:,:,1]<<8) + (mask_img[:,:,2]<<16)
# # Convert the 32bit integer color to 0,1, 2, ... labels.
colors, labels = np.unique(annotated_label, return_inverse=True)
n_labels = 2
#Setting up the CRF model
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
#Run Inference for 10 steps
Q = d.inference(10)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
return MAP.reshape((original_image.shape[0],original_image.shape[1]))
In [26]:
"""
Applying CRF on the predicted mask
"""
crf_output = []
for i in tqdm(range(X_test.shape[0])):
crf_output.append(crf(np.squeeze(X_test[i]),np.squeeze(final_preds_test[i])))
In [27]:
base_idx = 160
max_images = 32
grid_width = 4
grid_height = int(max_images / grid_width)
fig, axs = plt.subplots(grid_height, grid_width, figsize=(20, 20))
row = 0; col = 0;
for i in range(base_idx,base_idx+int(max_images)):
img = X_test[i].squeeze()
mask = crf_output[i].squeeze()
ax = axs[row, col];
ax.imshow(img, cmap="seismic")
ax.imshow(mask, alpha=0.5, cmap="Reds"); col+=1;
ax.set_yticklabels([]); ax.set_xticklabels([]);
if col >= grid_width:
col=0; row+=1;
In [28]:
threshold_best=threshold_best
pred_dict = {idx: RLenc(np.round(downsample(crf_output[i]))) for i, idx in enumerate(tqdm_notebook(test_df.img_id.values))}
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission_pinet_crf.csv')